Merge pull request #63 from albertsun/weibo

Weibo agents (WeiboUserAgent, WeiboPublishAgent)

Andrew Cantino 11 years ago
parent
commit
874531a26e

+ 1 - 0
Gemfile

@@ -37,6 +37,7 @@ gem 'wunderground'
37 37
 gem "twitter"
38 38
 gem 'twitter-stream', '>=0.1.16'
39 39
 gem 'em-http-request'
40
+gem 'weibo_2'
40 41
 
41 42
 platforms :ruby_18 do
42 43
   gem 'system_timer'

+ 17 - 0
Gemfile.lock

@@ -100,11 +100,13 @@ GEM
100 100
       rails (~> 3.0)
101 101
     haml (4.0.2)
102 102
       tilt
103
+    hashie (2.0.5)
103 104
     hike (1.2.1)
104 105
     http_parser.rb (0.5.3)
105 106
     httparty (0.10.2)
106 107
       multi_json (~> 1.0)
107 108
       multi_xml (>= 0.5.2)
109
+    httpauth (0.2.0)
108 110
     i18n (0.6.1)
109 111
     journey (1.0.4)
110 112
     jquery-rails (2.2.1)
@@ -137,6 +139,13 @@ GEM
137 139
     mysql2 (0.3.11)
138 140
     nested_form (0.3.2)
139 141
     nokogiri (1.5.9)
142
+    oauth2 (0.9.1)
143
+      faraday (~> 0.8)
144
+      httpauth (~> 0.1)
145
+      jwt (~> 0.1.4)
146
+      multi_json (~> 1.0)
147
+      multi_xml (~> 0.5)
148
+      rack (~> 1.2)
140 149
     orm_adapter (0.4.0)
141 150
     polyglot (0.3.3)
142 151
     pry (0.9.12)
@@ -187,6 +196,8 @@ GEM
187 196
     rdoc (3.12.2)
188 197
       json (~> 1.4)
189 198
     remotipart (1.0.5)
199
+    rest-client (1.6.7)
200
+      mime-types (>= 1.16)
190 201
     rr (1.0.4)
191 202
     rspec (2.13.0)
192 203
       rspec-core (~> 2.13.0)
@@ -253,6 +264,11 @@ GEM
253 264
     webmock (1.11.0)
254 265
       addressable (>= 2.2.7)
255 266
       crack (>= 0.3.2)
267
+    weibo_2 (0.1.4)
268
+      hashie (~> 2.0.4)
269
+      multi_json (~> 1.7.2)
270
+      oauth2 (~> 0.9.1)
271
+      rest-client (~> 1.6.7)
256 272
     wunderground (1.0.0)
257 273
       addressable
258 274
       httparty (> 0.6.0)
@@ -298,4 +314,5 @@ DEPENDENCIES
298 314
   typhoeus
299 315
   uglifier (>= 1.0.3)
300 316
   webmock
317
+  weibo_2
301 318
   wunderground

+ 92 - 0
app/models/agents/weibo_publish_agent.rb

@@ -0,0 +1,92 @@
1
+# encoding: utf-8 
2
+require "weibo_2"
3
+
4
+module Agents
5
+  class WeiboPublishAgent < Agent
6
+    cannot_be_scheduled!
7
+
8
+    description <<-MD
9
+      The WeiboPublishAgent publishes tweets from the events it receives.
10
+
11
+      You must first set up a Weibo app and generate an `acess_token` for the user to send statuses as.
12
+
13
+      Include that in options, along with the `app_key` and `app_secret` for your Weibo app. It's useful to also include the Weibo user id of the person to publish as.
14
+
15
+      You must also specify a `message_path` parameter: a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value to tweet.
16
+
17
+      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
18
+    MD
19
+
20
+    def validate_options
21
+      unless options[:uid].present? &&
22
+        options[:expected_update_period_in_days].present? &&
23
+        options[:app_key].present? &&
24
+        options[:app_secret].present? &&
25
+        options[:access_token].present?
26
+        errors.add(:base, "expected_update_period_in_days, uid, and access_token are required")
27
+      end
28
+    end
29
+
30
+    def working?
31
+      (event = event_created_within(options[:expected_update_period_in_days].to_i.days)) && event.payload.present? && event.payload[:success] == true
32
+    end
33
+
34
+    def default_options
35
+      {
36
+          :uid => "",
37
+          :access_token => "---",
38
+          :app_key => "---",
39
+          :app_secret => "---",
40
+          :expected_update_period_in_days => "10",
41
+          :message_path => "text"
42
+      }
43
+    end
44
+
45
+    def receive(incoming_events)
46
+      # if there are too many, dump a bunch to avoid getting rate limited
47
+      if incoming_events.count > 20
48
+        incoming_events = incoming_events.first(20)
49
+      end
50
+      incoming_events.each do |event|
51
+        tweet_text = Utils.value_at(event.payload, options[:message_path])
52
+        if event.agent.type == "Agents::TwitterUserAgent"
53
+          tweet_text = unwrap_tco_urls(tweet_text, event.payload)
54
+        end
55
+        begin
56
+          publish_tweet tweet_text
57
+          create_event :payload => {
58
+            :success => true,
59
+            :published_tweet => tweet_text,
60
+            :agent_id => event.agent_id,
61
+            :event_id => event.id
62
+          }
63
+        rescue OAuth2::Error => e
64
+          create_event :payload => {
65
+            :success => false,
66
+            :error => e.message,
67
+            :failed_tweet => tweet_text,
68
+            :agent_id => event.agent_id,
69
+            :event_id => event.id
70
+          }
71
+        end
72
+      end
73
+    end
74
+
75
+    def publish_tweet text
76
+      WeiboOAuth2::Config.api_key = options[:app_key] # WEIBO_APP_KEY
77
+      WeiboOAuth2::Config.api_secret = options[:app_secret] # WEIBO_APP_SECRET
78
+      client = WeiboOAuth2::Client.new
79
+      client.get_token_from_hash :access_token => options[:access_token]
80
+
81
+      client.statuses.update text
82
+    end
83
+
84
+    def unwrap_tco_urls text, tweet_json
85
+      tweet_json[:entities][:urls].each do |url|
86
+        text.gsub! url[:url], url[:expanded_url]
87
+      end
88
+      return text
89
+    end
90
+
91
+  end
92
+end

+ 120 - 0
app/models/agents/weibo_user_agent.rb

@@ -0,0 +1,120 @@
1
+# encoding: utf-8 
2
+require "weibo_2"
3
+
4
+module Agents
5
+  class WeiboUserAgent < Agent
6
+    cannot_receive_events!
7
+
8
+    description <<-MD
9
+      The WeiboUserAgent follows the timeline of a specified Weibo user. It uses this endpoint: http://open.weibo.com/wiki/2/statuses/user_timeline/en
10
+
11
+      You must first set up a Weibo app and generate an `acess_token` to authenticate with. Provide that, along with the `app_key` and `app_secret` for your Weibo app in the options.
12
+
13
+      Specify the `uid` of the Weibo user whose timeline you want to watch.
14
+
15
+      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
16
+    MD
17
+
18
+    event_description <<-MD
19
+      Events are the raw JSON provided by the Twitter API. Should look something like:
20
+
21
+        {
22
+          "created_at": "Tue May 31 17:46:55 +0800 2011",
23
+          "id": 11488058246,
24
+          "text": "求关注。",
25
+          "source": "<a href=\"http://weibo.com\" rel=\"nofollow\">新浪微博</a>",
26
+          "favorited": false,
27
+          "truncated": false,
28
+          "in_reply_to_status_id": "",
29
+          "in_reply_to_user_id": "",
30
+          "in_reply_to_screen_name": "",
31
+          "geo": null,
32
+          "mid": "5612814510546515491",
33
+          "reposts_count": 8,
34
+          "comments_count": 9,
35
+          "annotations": [],
36
+          "user": {
37
+              "id": 1404376560,
38
+              "screen_name": "zaku",
39
+              "name": "zaku",
40
+              "province": "11",
41
+              "city": "5",
42
+              "location": "北京 朝阳区",
43
+              "description": "人生五十年,乃如梦如幻;有生斯有死,壮士复何憾。",
44
+              "url": "http://blog.sina.com.cn/zaku",
45
+              "profile_image_url": "http://tp1.sinaimg.cn/1404376560/50/0/1",
46
+              "domain": "zaku",
47
+              "gender": "m",
48
+              "followers_count": 1204,
49
+              "friends_count": 447,
50
+              "statuses_count": 2908,
51
+              "favourites_count": 0,
52
+              "created_at": "Fri Aug 28 00:00:00 +0800 2009",
53
+              "following": false,
54
+              "allow_all_act_msg": false,
55
+              "remark": "",
56
+              "geo_enabled": true,
57
+              "verified": false,
58
+              "allow_all_comment": true,
59
+              "avatar_large": "http://tp1.sinaimg.cn/1404376560/180/0/1",
60
+              "verified_reason": "",
61
+              "follow_me": false,
62
+              "online_status": 0,
63
+              "bi_followers_count": 215
64
+          }
65
+        }
66
+    MD
67
+
68
+    default_schedule "every_1h"
69
+
70
+    def validate_options
71
+      unless options[:uid].present? &&
72
+        options[:expected_update_period_in_days].present? &&
73
+        options[:app_key].present? &&
74
+        options[:app_secret].present? &&
75
+        options[:access_token].present?
76
+        errors.add(:base, "expected_update_period_in_days, uid, app_key, app_secret and access_token are required")
77
+      end
78
+    end
79
+
80
+    def working?
81
+      (event = event_created_within(options[:expected_update_period_in_days].to_i.days)) && event.payload.present?
82
+    end
83
+
84
+    def default_options
85
+      {
86
+          :uid => "",
87
+          :access_token => "---",
88
+          :app_key => "---",
89
+          :app_secret => "---",
90
+          :expected_update_period_in_days => "2"
91
+      }
92
+    end
93
+
94
+    def check
95
+      WeiboOAuth2::Config.api_key = options[:app_key] # WEIBO_APP_KEY
96
+      WeiboOAuth2::Config.api_secret = options[:app_secret] # WEIBO_APP_SECRET
97
+      client = WeiboOAuth2::Client.new
98
+      client.get_token_from_hash :access_token => options[:access_token]
99
+
100
+
101
+      since_id = memory[:since_id] || nil
102
+      opts = {:uid => options[:uid].to_i}
103
+      opts.merge! :since_id => since_id unless since_id.nil?
104
+
105
+      # http://open.weibo.com/wiki/2/statuses/user_timeline/en
106
+      resp = client.statuses.user_timeline opts
107
+      if resp[:statuses]
108
+
109
+
110
+        resp[:statuses].each do |status|
111
+          memory[:since_id] = status.id if !memory[:since_id] || (status.id > memory[:since_id])
112
+
113
+          create_event :payload => status.as_json
114
+        end
115
+      end
116
+
117
+      save!
118
+    end
119
+  end
120
+end

+ 128 - 0
spec/data_fixtures/one_tweet.json

@@ -0,0 +1,128 @@
1
+{ 
2
+    "created_at": "Sat Jun 15 20:10:32 +0000 2013", 
3
+    "id": 345996769290752000, 
4
+    "id_str": "345996769290752000", 
5
+    "text": "Crytoscape is a graph manipulation library for JS.  Impressive.  http://t.co/KQFGZWvkSs", 
6
+    "source": "<a href=\"http://tapbots.com/software/tweetbot/mac\" rel=\"nofollow\">Tweetbot for Mac</a>", 
7
+    "truncated": false, 
8
+    "in_reply_to_status_id": null, 
9
+    "in_reply_to_status_id_str": null, 
10
+    "in_reply_to_user_id": null, 
11
+    "in_reply_to_user_id_str": null, 
12
+    "in_reply_to_screen_name": null, 
13
+    "user": { 
14
+        "id": 9813372, 
15
+        "id_str": "9813372", 
16
+        "name": "Andrew Cantino", 
17
+        "screen_name": "tectonic", 
18
+        "location": "San Francisco, CA", 
19
+        "description": "Experimentalist, web developer, and VP of Engineering at @Mavenlink.", 
20
+        "url": "http://t.co/SKoQz7cOVI", 
21
+        "entities": { 
22
+            "url": { 
23
+                "urls": [ 
24
+                    { 
25
+                        "url": "http://t.co/SKoQz7cOVI", 
26
+                        "expanded_url": "http://andrewcantino.com", 
27
+                        "display_url": "andrewcantino.com", 
28
+                        "indices": [ 
29
+                            0, 
30
+                            22 
31
+                        ] 
32
+                    } 
33
+                ] 
34
+            }, 
35
+            "description": { 
36
+                "urls": [] 
37
+            } 
38
+        }, 
39
+        "protected": false, 
40
+        "followers_count": 1056, 
41
+        "friends_count": 492, 
42
+        "listed_count": 37, 
43
+        "created_at": "Wed Oct 31 03:16:39 +0000 2007", 
44
+        "favourites_count": 151, 
45
+        "utc_offset": -28800, 
46
+        "time_zone": "Pacific Time (US & Canada)", 
47
+        "geo_enabled": true, 
48
+        "verified": false, 
49
+        "statuses_count": 3628, 
50
+        "lang": "en", 
51
+        "contributors_enabled": false, 
52
+        "is_translator": false, 
53
+        "profile_background_color": "352726", 
54
+        "profile_background_image_url": "http://a0.twimg.com/images/themes/theme5/bg.gif", 
55
+        "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme5/bg.gif", 
56
+        "profile_background_tile": false, 
57
+        "profile_image_url": "http://a0.twimg.com/profile_images/1694984565/me-right_normal.jpg", 
58
+        "profile_image_url_https": "https://si0.twimg.com/profile_images/1694984565/me-right_normal.jpg", 
59
+        "profile_link_color": "D02B55", 
60
+        "profile_sidebar_border_color": "829D5E", 
61
+        "profile_sidebar_fill_color": "99CC33", 
62
+        "profile_text_color": "3E4415", 
63
+        "profile_use_background_image": true, 
64
+        "default_profile": false, 
65
+        "default_profile_image": false, 
66
+        "following": true, 
67
+        "follow_request_sent": false, 
68
+        "notifications": null 
69
+    }, 
70
+    "geo": null, 
71
+    "coordinates": null, 
72
+    "place": { 
73
+        "id": "866269c983527d5a", 
74
+        "url": "https://api.twitter.com/1.1/geo/id/866269c983527d5a.json", 
75
+        "place_type": "neighborhood", 
76
+        "name": "Ashbury Heights", 
77
+        "full_name": "Ashbury Heights, San Francisco", 
78
+        "country_code": "US", 
79
+        "country": "United States", 
80
+        "bounding_box": { 
81
+            "type": "Polygon", 
82
+            "coordinates": [ 
83
+                [ 
84
+                    [ 
85
+                        -122.45778216, 
86
+                        37.75932999 
87
+                    ], 
88
+                    [ 
89
+                        -122.44248216, 
90
+                        37.75932999 
91
+                    ], 
92
+                    [ 
93
+                        -122.44248216, 
94
+                        37.767528989999995 
95
+                    ], 
96
+                    [ 
97
+                        -122.45778216, 
98
+                        37.767528989999995 
99
+                    ] 
100
+                ] 
101
+            ] 
102
+        }, 
103
+        "attributes": {} 
104
+    }, 
105
+    "contributors": null, 
106
+    "retweet_count": 0, 
107
+    "favorite_count": 2, 
108
+    "entities": { 
109
+        "hashtags": [], 
110
+        "symbols": [], 
111
+        "urls": [ 
112
+            { 
113
+                "url": "http://t.co/KQFGZWvkSs", 
114
+                "expanded_url": "http://cytoscape.github.io/cytoscape.js/", 
115
+                "display_url": "cytoscape.github.io/cytoscape.js/", 
116
+                "indices": [ 
117
+                    65, 
118
+                    87 
119
+                ] 
120
+            } 
121
+        ], 
122
+        "user_mentions": [] 
123
+    }, 
124
+    "favorited": false, 
125
+    "retweeted": false, 
126
+    "possibly_sensitive": false, 
127
+    "lang": "en" 
128
+}

+ 52 - 0
spec/data_fixtures/one_weibo.json

@@ -0,0 +1,52 @@
1
+{
2
+    "statuses": [
3
+      {
4
+        "created_at": "Tue May 31 17:46:55 +0800 2011",
5
+        "id": 11488058246,
6
+        "text": "求关注。",
7
+        "source": "<a href=\"http://weibo.com\" rel=\"nofollow\">新浪微博</a>",
8
+        "favorited": false,
9
+        "truncated": false,
10
+        "in_reply_to_status_id": "",
11
+        "in_reply_to_user_id": "",
12
+        "in_reply_to_screen_name": "",
13
+        "geo": null,
14
+        "mid": "5612814510546515491",
15
+        "reposts_count": 8,
16
+        "comments_count": 9,
17
+        "annotations": [],
18
+        "user": {
19
+            "id": 1404376560,
20
+            "screen_name": "zaku",
21
+            "name": "zaku",
22
+            "province": "11",
23
+            "city": "5",
24
+            "location": "北京 朝阳区",
25
+            "description": "人生五十年,乃如梦如幻;有生斯有死,壮士复何憾。",
26
+            "url": "http://blog.sina.com.cn/zaku",
27
+            "profile_image_url": "http://tp1.sinaimg.cn/1404376560/50/0/1",
28
+            "domain": "zaku",
29
+            "gender": "m",
30
+            "followers_count": 1204,
31
+            "friends_count": 447,
32
+            "statuses_count": 2908,
33
+            "favourites_count": 0,
34
+            "created_at": "Fri Aug 28 00:00:00 +0800 2009",
35
+            "following": false,
36
+            "allow_all_act_msg": false,
37
+            "remark": "",
38
+            "geo_enabled": true,
39
+            "verified": false,
40
+            "allow_all_comment": true,
41
+            "avatar_large": "http://tp1.sinaimg.cn/1404376560/180/0/1",
42
+            "verified_reason": "",
43
+            "follow_me": false,
44
+            "online_status": 0,
45
+            "bi_followers_count": 215
46
+        }
47
+    }
48
+  ],
49
+   "previous_cursor": 0,
50
+    "next_cursor": 11488013766,
51
+    "total_number": 81655
52
+}

+ 13 - 0
spec/fixtures/agents.yml

@@ -79,3 +79,16 @@ bob_rain_notifier_agent:
79 79
                   }],
80 80
                  :message => "Just so you know, it looks like '<conditions>' tomorrow in <zipcode>"
81 81
                }.to_yaml.inspect %>
82
+
83
+bob_twitter_user_agent:
84
+  type: Agents::TwitterUserAgent
85
+  user: bob
86
+  name: "Bob's Twitter User Watcher"
87
+  options: <%= {
88
+      :username => "tectonic",
89
+      :expected_update_period_in_days => "2",
90
+      :consumer_key => "---",
91
+      :consumer_secret => "---",
92
+      :oauth_token => "---",
93
+      :oauth_token_secret => "---"
94
+    }.to_yaml.inspect %>

+ 70 - 0
spec/models/agents/weibo_publish_agent_spec.rb

@@ -0,0 +1,70 @@
1
+# encoding: utf-8 
2
+require 'spec_helper'
3
+
4
+describe Agents::WeiboPublishAgent do
5
+  before do
6
+    @opts = {
7
+      :uid => "1234567",
8
+      :expected_update_period_in_days => "2",
9
+      :app_key => "---",
10
+      :app_secret => "---",
11
+      :access_token => "---",
12
+      :message_path => "text"
13
+    }
14
+
15
+    @checker = Agents::WeiboPublishAgent.new(:name => "Weibo Publisher", :options => @opts)
16
+    @checker.user = users(:bob)
17
+    @checker.save!
18
+
19
+    @event = Event.new
20
+    @event.agent = agents(:bob_weather_agent)
21
+    @event.payload = { :text => 'Gonna rain..' }
22
+    @event.save!
23
+
24
+    @sent_messages = []
25
+    stub.any_instance_of(Agents::WeiboPublishAgent).publish_tweet { |message| @sent_messages << message}
26
+  end
27
+
28
+  describe '#receive' do
29
+    it 'should publish any payload it receives' do
30
+      event1 = Event.new
31
+      event1.agent = agents(:bob_rain_notifier_agent)
32
+      event1.payload = { :text => 'Gonna rain..' }
33
+      event1.save!
34
+
35
+      event2 = Event.new
36
+      event2.agent = agents(:bob_weather_agent)
37
+      event2.payload = { :text => 'More payload' }
38
+      event2.save!
39
+
40
+      Agents::WeiboPublishAgent.async_receive(@checker.id, [event1.id, event2.id])
41
+      @sent_messages.count.should eq(2)
42
+      @checker.events.count.should eq(2)
43
+    end
44
+  end
45
+
46
+  describe '#receive a tweet' do
47
+    it 'should publish a tweet after expanding any t.co urls' do
48
+      event = Event.new
49
+      event.agent = agents(:bob_twitter_user_agent)
50
+      event.payload = JSON.parse(File.read(Rails.root.join("spec/data_fixtures/one_tweet.json")))
51
+      event.save!
52
+
53
+      Agents::WeiboPublishAgent.async_receive(@checker.id, [event.id])
54
+      @sent_messages.count.should eq(1)
55
+      @checker.events.count.should eq(1)
56
+      @sent_messages.first.include?("t.co").should_not be_true
57
+    end
58
+  end
59
+
60
+  describe '#working?' do
61
+    it 'checks if events have been received within the expected receive period' do
62
+      @checker.should_not be_working # No events received
63
+      Agents::WeiboPublishAgent.async_receive(@checker.id, [@event.id])
64
+      @checker.reload.should be_working # Just received events
65
+      two_days_from_now = 2.days.from_now
66
+      stub(Time).now { two_days_from_now }
67
+      @checker.reload.should_not be_working # More time has passed than the expected receive period without any new events
68
+    end
69
+  end
70
+end

+ 28 - 0
spec/models/agents/weibo_user_agent_spec.rb

@@ -0,0 +1,28 @@
1
+# encoding: utf-8 
2
+require 'spec_helper'
3
+
4
+describe Agents::WeiboUserAgent do
5
+  before do
6
+    # intercept the twitter API request for @tectonic's user profile
7
+    stub_request(:any, /api.weibo.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/one_weibo.json")), :status => 200)
8
+  
9
+    @opts = {
10
+      :uid => "123456",
11
+      :expected_update_period_in_days => "2",
12
+      :app_key => "asdfe",
13
+      :app_secret => "asdfe",
14
+      :access_token => "asdfe"
15
+    }
16
+
17
+    @checker = Agents::WeiboUserAgent.new(:name => "123456 fetcher", :options => @opts)
18
+    @checker.user = users(:bob)
19
+    @checker.save!
20
+  end
21
+
22
+  describe "#check" do
23
+    it "should check for changes" do
24
+      lambda { @checker.check }.should change { Event.count }.by(1)
25
+    end
26
+  end
27
+
28
+end